{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c298b1c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('frames_salah_trial.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a4ff9269",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>query</th>\n",
       "      <th>result</th>\n",
       "      <th>project_id</th>\n",
       "      <th>execution_time</th>\n",
       "      <th>node_count</th>\n",
       "      <th>completion_status</th>\n",
       "      <th>eval_timestamp</th>\n",
       "      <th>server_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>If my future wife has the same first name as t...</td>\n",
       "      <td>Jane Ballou</td>\n",
       "      <td>If my future wife has the same first name as t...</td>\n",
       "      <td>I will not repeat any of the reasoning steps ...</td>\n",
       "      <td>5ee34f33-7656-45db-9a1a-a3faf6d8c77a</td>\n",
       "      <td>93.721762</td>\n",
       "      <td>5</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Imagine there is a building called Bronte towe...</td>\n",
       "      <td>37th</td>\n",
       "      <td>Imagine there is a building called Bronte towe...</td>\n",
       "      <td>\\n\\nThe Charlotte Brontë book published in 184...</td>\n",
       "      <td>82288b5e-365a-4a90-aab9-0b7815900094</td>\n",
       "      <td>248.959409</td>\n",
       "      <td>5</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How many years earlier would Punxsutawney Phil...</td>\n",
       "      <td>87</td>\n",
       "      <td>How many years earlier would Punxsutawney Phil...</td>\n",
       "      <td>\\n\\nBased on the research conducted, Punxsutaw...</td>\n",
       "      <td>e62cd668-ed51-44c8-a954-02013e1e9898</td>\n",
       "      <td>180.489119</td>\n",
       "      <td>4</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>As of August 1, 2024, which country were holde...</td>\n",
       "      <td>France</td>\n",
       "      <td>As of August 1, 2024, which country were holde...</td>\n",
       "      <td>\\nAs of August 1, 2024, the country that held ...</td>\n",
       "      <td>58ea534f-96ad-45af-93cf-bdfc1367e266</td>\n",
       "      <td>119.789040</td>\n",
       "      <td>4</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What is the name of the vocalist from the firs...</td>\n",
       "      <td>Jens Kidman</td>\n",
       "      <td>What is the name of the vocalist from the firs...</td>\n",
       "      <td>I will use the information from my reasoning ...</td>\n",
       "      <td>efeb922d-8b8a-4b7d-9c28-ac6af5b2557f</td>\n",
       "      <td>115.867239</td>\n",
       "      <td>4</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            question       answer  \\\n",
       "0  If my future wife has the same first name as t...  Jane Ballou   \n",
       "1  Imagine there is a building called Bronte towe...         37th   \n",
       "2  How many years earlier would Punxsutawney Phil...           87   \n",
       "3  As of August 1, 2024, which country were holde...       France   \n",
       "4  What is the name of the vocalist from the firs...  Jens Kidman   \n",
       "\n",
       "                                               query  \\\n",
       "0  If my future wife has the same first name as t...   \n",
       "1  Imagine there is a building called Bronte towe...   \n",
       "2  How many years earlier would Punxsutawney Phil...   \n",
       "3  As of August 1, 2024, which country were holde...   \n",
       "4  What is the name of the vocalist from the firs...   \n",
       "\n",
       "                                              result  \\\n",
       "0   I will not repeat any of the reasoning steps ...   \n",
       "1  \\n\\nThe Charlotte Brontë book published in 184...   \n",
       "2  \\n\\nBased on the research conducted, Punxsutaw...   \n",
       "3  \\nAs of August 1, 2024, the country that held ...   \n",
       "4   I will use the information from my reasoning ...   \n",
       "\n",
       "                             project_id  execution_time  node_count  \\\n",
       "0  5ee34f33-7656-45db-9a1a-a3faf6d8c77a       93.721762           5   \n",
       "1  82288b5e-365a-4a90-aab9-0b7815900094      248.959409           5   \n",
       "2  e62cd668-ed51-44c8-a954-02013e1e9898      180.489119           4   \n",
       "3  58ea534f-96ad-45af-93cf-bdfc1367e266      119.789040           4   \n",
       "4  efeb922d-8b8a-4b7d-9c28-ac6af5b2557f      115.867239           4   \n",
       "\n",
       "  completion_status              eval_timestamp             server_url  \n",
       "0         completed  2025-07-11T18:30:27.053409  http://localhost:5000  \n",
       "1         completed  2025-07-11T18:30:27.053409  http://localhost:5000  \n",
       "2         completed  2025-07-11T18:30:27.053409  http://localhost:5000  \n",
       "3         completed  2025-07-11T18:30:27.053409  http://localhost:5000  \n",
       "4         completed  2025-07-11T18:30:27.053409  http://localhost:5000  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b05708b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = df.sample(1)\n",
    "question = sample['question'].values[0]\n",
    "answer = sample['result'].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dc0bdc52",
   "metadata": {},
   "outputs": [],
   "source": [
    "from litellm import completion\n",
    "def extract_final_answer(query: str, result: str, model: str= \"openrouter/google/gemini-2.5-flash\") -> str:\n",
    "    \"\"\"Extract the final answer from a verbose result using LiteLLM.\"\"\"\n",
    "        \n",
    "    prompt = f\"\"\"Given the following query and verbose result, extract ONLY the final answer. \n",
    "    Return just the exact final answer with no additional text, explanations, or formatting.\n",
    "\n",
    "    Query: {query}\n",
    "\n",
    "    Verbose Result: {result}\n",
    "\n",
    "    Final Answer:\"\"\"\n",
    "\n",
    "    response = completion(\n",
    "        model=model,\n",
    "        messages=[{\"role\": \"user\", \"content\": prompt}],\n",
    "        temperature=0.4,\n",
    "    )\n",
    "\n",
    "    final_answer = response.choices[0].message.content.strip()\n",
    "    return final_answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "32a54f94",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/salahalzubi/cursor_projects/SentientResearchAgent/.venv/lib/python3.12/site-packages/pydantic/main.py:463: UserWarning: Pydantic serializer warnings:\n",
      "  PydanticSerializationUnexpectedValue(Expected 9 fields but got 5: Expected `Message` - serialized value may not be as expected [input_value=Message(content='95.35%',...one, 'reasoning': None}), input_type=Message])\n",
      "  PydanticSerializationUnexpectedValue(Expected `StreamingChoices` - serialized value may not be as expected [input_value=Choices(finish_reason='st...finish_reason': 'STOP'}), input_type=Choices])\n",
      "  return self.__pydantic_serializer__.to_python(\n"
     ]
    }
   ],
   "source": [
    "final_answer = extract_final_answer(question, answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "db70a2da",
   "metadata": {},
   "outputs": [],
   "source": [
    "extracted_df = pd.read_csv(\"frames_salah_trial_extracted.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "463df119",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>extracted_answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>If my future wife has the same first name as t...</td>\n",
       "      <td>Jane Ballou</td>\n",
       "      <td>Jane Ballou</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Imagine there is a building called Bronte towe...</td>\n",
       "      <td>37th</td>\n",
       "      <td>35th-tallest building in New York City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How many years earlier would Punxsutawney Phil...</td>\n",
       "      <td>87</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>As of August 1, 2024, which country were holde...</td>\n",
       "      <td>France</td>\n",
       "      <td>France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What is the name of the vocalist from the firs...</td>\n",
       "      <td>Jens Kidman</td>\n",
       "      <td>Mitch Lucker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>As of August 3, 2024, what is the sum of the b...</td>\n",
       "      <td>5980</td>\n",
       "      <td>1909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>Rosie Ruiz was disqualified from the Boston Ma...</td>\n",
       "      <td>Jacqueline Gareau placed first in the 1988 Gra...</td>\n",
       "      <td>1st</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>On Nov 15, 2017 a painting was sold for US $45...</td>\n",
       "      <td>1766</td>\n",
       "      <td>1766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>A disease that had millions of dollars raised ...</td>\n",
       "      <td>1981</td>\n",
       "      <td>1981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>As of July 1, 2023, what is the total number o...</td>\n",
       "      <td>43</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              question  \\\n",
       "0    If my future wife has the same first name as t...   \n",
       "1    Imagine there is a building called Bronte towe...   \n",
       "2    How many years earlier would Punxsutawney Phil...   \n",
       "3    As of August 1, 2024, which country were holde...   \n",
       "4    What is the name of the vocalist from the firs...   \n",
       "..                                                 ...   \n",
       "145  As of August 3, 2024, what is the sum of the b...   \n",
       "146  Rosie Ruiz was disqualified from the Boston Ma...   \n",
       "147  On Nov 15, 2017 a painting was sold for US $45...   \n",
       "148  A disease that had millions of dollars raised ...   \n",
       "149  As of July 1, 2023, what is the total number o...   \n",
       "\n",
       "                                                answer  \\\n",
       "0                                          Jane Ballou   \n",
       "1                                                 37th   \n",
       "2                                                   87   \n",
       "3                                               France   \n",
       "4                                          Jens Kidman   \n",
       "..                                                 ...   \n",
       "145                                               5980   \n",
       "146  Jacqueline Gareau placed first in the 1988 Gra...   \n",
       "147                                               1766   \n",
       "148                                               1981   \n",
       "149                                                 43   \n",
       "\n",
       "                           extracted_answer  \n",
       "0                               Jane Ballou  \n",
       "1    35th-tallest building in New York City  \n",
       "2                                        87  \n",
       "3                                    France  \n",
       "4                              Mitch Lucker  \n",
       "..                                      ...  \n",
       "145                                    1909  \n",
       "146                                     1st  \n",
       "147                                    1766  \n",
       "148                                    1981  \n",
       "149                                      41  \n",
       "\n",
       "[150 rows x 3 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extracted_df[['question', 'answer', 'extracted_answer']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "40ac177f",
   "metadata": {},
   "outputs": [],
   "source": [
    "graded_df = pd.read_csv(\"frames_salah_trial_extracted_graded.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ac6745d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "grade\n",
       "1    0.586667\n",
       "0    0.413333\n",
       "Name: proportion, dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graded_df.grade.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cd180b50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>query</th>\n",
       "      <th>result</th>\n",
       "      <th>project_id</th>\n",
       "      <th>execution_time</th>\n",
       "      <th>node_count</th>\n",
       "      <th>completion_status</th>\n",
       "      <th>eval_timestamp</th>\n",
       "      <th>server_url</th>\n",
       "      <th>extracted_answer</th>\n",
       "      <th>grade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Imagine there is a building called Bronte towe...</td>\n",
       "      <td>37th</td>\n",
       "      <td>Imagine there is a building called Bronte towe...</td>\n",
       "      <td>\\n\\nThe Charlotte Brontë book published in 184...</td>\n",
       "      <td>82288b5e-365a-4a90-aab9-0b7815900094</td>\n",
       "      <td>248.959409</td>\n",
       "      <td>5</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "      <td>35th-tallest building in New York City</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What is the name of the vocalist from the firs...</td>\n",
       "      <td>Jens Kidman</td>\n",
       "      <td>What is the name of the vocalist from the firs...</td>\n",
       "      <td>I will use the information from my reasoning ...</td>\n",
       "      <td>efeb922d-8b8a-4b7d-9c28-ac6af5b2557f</td>\n",
       "      <td>115.867239</td>\n",
       "      <td>4</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "      <td>Mitch Lucker</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>I have an element in mind and would like you t...</td>\n",
       "      <td>Mendelevium is named after Dmitri Mendeleev.</td>\n",
       "      <td>I have an element in mind and would like you t...</td>\n",
       "      <td>\\nThe element with atomic number 106 is **seab...</td>\n",
       "      <td>3d40222d-aebd-4c91-a5af-d85a399fcc1b</td>\n",
       "      <td>138.037470</td>\n",
       "      <td>5</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "      <td>Glenn T. Seaborg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>A general motors vehicle is named after the la...</td>\n",
       "      <td>4</td>\n",
       "      <td>A general motors vehicle is named after the la...</td>\n",
       "      <td>No output generated</td>\n",
       "      <td>7bf18bf3-7d9a-405b-992b-3cf5731f9a5f</td>\n",
       "      <td>198.480795</td>\n",
       "      <td>6</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>In Slovakia there is a well known Film Festiva...</td>\n",
       "      <td>Roudnice nad Labem</td>\n",
       "      <td>In Slovakia there is a well known Film Festiva...</td>\n",
       "      <td>Planned 3 sub-tasks: Identify the film that wo...</td>\n",
       "      <td>6a5b6708-35b2-409e-8b33-a6d5a8b9addc</td>\n",
       "      <td>43.298700</td>\n",
       "      <td>4</td>\n",
       "      <td>running</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "      <td>Paris</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>Suppose Egon Sendler's book \"Les mystères du C...</td>\n",
       "      <td>51 years.</td>\n",
       "      <td>Suppose Egon Sendler's book \"Les mystères du C...</td>\n",
       "      <td>I will use the direct answer format.\\nBased o...</td>\n",
       "      <td>3d70fcb7-c39f-4169-a4c1-125d7ea72c7a</td>\n",
       "      <td>178.883230</td>\n",
       "      <td>4</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "      <td>-28 years</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>As of August 5, 2024, what is the name of the ...</td>\n",
       "      <td>The Sherman Antitrust Act</td>\n",
       "      <td>As of August 5, 2024, what is the name of the ...</td>\n",
       "      <td>\\nAs of August 5, 2024, the federal law that B...</td>\n",
       "      <td>e019e7b1-7ab0-47f4-8253-7095f1948a70</td>\n",
       "      <td>104.134771</td>\n",
       "      <td>4</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "      <td>National Labor Relations Act (NLRA)</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>As of August 3, 2024, what is the sum of the b...</td>\n",
       "      <td>5980</td>\n",
       "      <td>As of August 3, 2024, what is the sum of the b...</td>\n",
       "      <td>\\nBased on the provided sub-task results, the ...</td>\n",
       "      <td>bdd48db0-aea6-4b24-9753-3da3bbbe80b5</td>\n",
       "      <td>138.472103</td>\n",
       "      <td>4</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "      <td>1909</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>Rosie Ruiz was disqualified from the Boston Ma...</td>\n",
       "      <td>Jacqueline Gareau placed first in the 1988 Gra...</td>\n",
       "      <td>Rosie Ruiz was disqualified from the Boston Ma...</td>\n",
       "      <td>\\nAfter Rosie Ruiz's disqualification from the...</td>\n",
       "      <td>e4934e95-8904-42d8-a5df-1956ea747c43</td>\n",
       "      <td>128.335218</td>\n",
       "      <td>4</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "      <td>1st</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>As of July 1, 2023, what is the total number o...</td>\n",
       "      <td>43</td>\n",
       "      <td>As of July 1, 2023, what is the total number o...</td>\n",
       "      <td>I will use my reasoning and the user-provided...</td>\n",
       "      <td>0869d2c3-46a5-4f04-8399-69a60505985b</td>\n",
       "      <td>132.456926</td>\n",
       "      <td>4</td>\n",
       "      <td>completed</td>\n",
       "      <td>2025-07-11T18:30:27.053409</td>\n",
       "      <td>http://localhost:5000</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>62 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              question  \\\n",
       "1    Imagine there is a building called Bronte towe...   \n",
       "4    What is the name of the vocalist from the firs...   \n",
       "6    I have an element in mind and would like you t...   \n",
       "8    A general motors vehicle is named after the la...   \n",
       "15   In Slovakia there is a well known Film Festiva...   \n",
       "..                                                 ...   \n",
       "141  Suppose Egon Sendler's book \"Les mystères du C...   \n",
       "142  As of August 5, 2024, what is the name of the ...   \n",
       "145  As of August 3, 2024, what is the sum of the b...   \n",
       "146  Rosie Ruiz was disqualified from the Boston Ma...   \n",
       "149  As of July 1, 2023, what is the total number o...   \n",
       "\n",
       "                                                answer  \\\n",
       "1                                                 37th   \n",
       "4                                          Jens Kidman   \n",
       "6         Mendelevium is named after Dmitri Mendeleev.   \n",
       "8                                                    4   \n",
       "15                                  Roudnice nad Labem   \n",
       "..                                                 ...   \n",
       "141                                          51 years.   \n",
       "142                          The Sherman Antitrust Act   \n",
       "145                                               5980   \n",
       "146  Jacqueline Gareau placed first in the 1988 Gra...   \n",
       "149                                                 43   \n",
       "\n",
       "                                                 query  \\\n",
       "1    Imagine there is a building called Bronte towe...   \n",
       "4    What is the name of the vocalist from the firs...   \n",
       "6    I have an element in mind and would like you t...   \n",
       "8    A general motors vehicle is named after the la...   \n",
       "15   In Slovakia there is a well known Film Festiva...   \n",
       "..                                                 ...   \n",
       "141  Suppose Egon Sendler's book \"Les mystères du C...   \n",
       "142  As of August 5, 2024, what is the name of the ...   \n",
       "145  As of August 3, 2024, what is the sum of the b...   \n",
       "146  Rosie Ruiz was disqualified from the Boston Ma...   \n",
       "149  As of July 1, 2023, what is the total number o...   \n",
       "\n",
       "                                                result  \\\n",
       "1    \\n\\nThe Charlotte Brontë book published in 184...   \n",
       "4     I will use the information from my reasoning ...   \n",
       "6    \\nThe element with atomic number 106 is **seab...   \n",
       "8                                  No output generated   \n",
       "15   Planned 3 sub-tasks: Identify the film that wo...   \n",
       "..                                                 ...   \n",
       "141   I will use the direct answer format.\\nBased o...   \n",
       "142  \\nAs of August 5, 2024, the federal law that B...   \n",
       "145  \\nBased on the provided sub-task results, the ...   \n",
       "146  \\nAfter Rosie Ruiz's disqualification from the...   \n",
       "149   I will use my reasoning and the user-provided...   \n",
       "\n",
       "                               project_id  execution_time  node_count  \\\n",
       "1    82288b5e-365a-4a90-aab9-0b7815900094      248.959409           5   \n",
       "4    efeb922d-8b8a-4b7d-9c28-ac6af5b2557f      115.867239           4   \n",
       "6    3d40222d-aebd-4c91-a5af-d85a399fcc1b      138.037470           5   \n",
       "8    7bf18bf3-7d9a-405b-992b-3cf5731f9a5f      198.480795           6   \n",
       "15   6a5b6708-35b2-409e-8b33-a6d5a8b9addc       43.298700           4   \n",
       "..                                    ...             ...         ...   \n",
       "141  3d70fcb7-c39f-4169-a4c1-125d7ea72c7a      178.883230           4   \n",
       "142  e019e7b1-7ab0-47f4-8253-7095f1948a70      104.134771           4   \n",
       "145  bdd48db0-aea6-4b24-9753-3da3bbbe80b5      138.472103           4   \n",
       "146  e4934e95-8904-42d8-a5df-1956ea747c43      128.335218           4   \n",
       "149  0869d2c3-46a5-4f04-8399-69a60505985b      132.456926           4   \n",
       "\n",
       "    completion_status              eval_timestamp             server_url  \\\n",
       "1           completed  2025-07-11T18:30:27.053409  http://localhost:5000   \n",
       "4           completed  2025-07-11T18:30:27.053409  http://localhost:5000   \n",
       "6           completed  2025-07-11T18:30:27.053409  http://localhost:5000   \n",
       "8           completed  2025-07-11T18:30:27.053409  http://localhost:5000   \n",
       "15            running  2025-07-11T18:30:27.053409  http://localhost:5000   \n",
       "..                ...                         ...                    ...   \n",
       "141         completed  2025-07-11T18:30:27.053409  http://localhost:5000   \n",
       "142         completed  2025-07-11T18:30:27.053409  http://localhost:5000   \n",
       "145         completed  2025-07-11T18:30:27.053409  http://localhost:5000   \n",
       "146         completed  2025-07-11T18:30:27.053409  http://localhost:5000   \n",
       "149         completed  2025-07-11T18:30:27.053409  http://localhost:5000   \n",
       "\n",
       "                           extracted_answer  grade  \n",
       "1    35th-tallest building in New York City      0  \n",
       "4                              Mitch Lucker      0  \n",
       "6                          Glenn T. Seaborg      0  \n",
       "8                                        12      0  \n",
       "15                                    Paris      0  \n",
       "..                                      ...    ...  \n",
       "141                               -28 years      0  \n",
       "142     National Labor Relations Act (NLRA)      0  \n",
       "145                                    1909      0  \n",
       "146                                     1st      0  \n",
       "149                                      41      0  \n",
       "\n",
       "[62 rows x 12 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graded_df[graded_df['grade'] == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "936d6c62",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
